{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "import numpy as np\n",
    "from collections import Counter\n",
    "import tensorflow as tf\n",
    "import random\n",
    "\n",
    "import os\n",
    "import pickle\n",
    "import re\n",
    "from tensorflow.python.ops import math_ops"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "# 原始数据文档\n",
    "* https://tianchi.aliyun.com/datalab/dataSet.html?dataId=408\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 样本骨架 数据结构\n",
    "<img src=\"assets/sample_skeleton.jpg\"/>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Common Feature 数据结构\n",
    "<img src=\"assets/common_feature.jpg\"/>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 购买样本全部保留\n",
    "### 训练骨架"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "'module' object is not callable",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-2-5a2bb33039dc>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0ma\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mrandom\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m: 'module' object is not callable"
     ]
    }
   ],
   "source": [
    "a = random()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "current_index: 1000000\n",
      "current_index: 2000000\n",
      "current_index: 3000000\n",
      "current_index: 4000000\n",
      "current_index: 5000000\n",
      "current_index: 6000000\n",
      "current_index: 7000000\n",
      "current_index: 8000000\n",
      "current_index: 9000000\n",
      "current_index: 10000000\n",
      "current_index: 11000000\n",
      "current_index: 12000000\n",
      "current_index: 13000000\n",
      "current_index: 14000000\n",
      "current_index: 15000000\n",
      "current_index: 16000000\n",
      "current_index: 17000000\n",
      "current_index: 18000000\n",
      "current_index: 19000000\n",
      "current_index: 20000000\n",
      "current_index: 21000000\n",
      "current_index: 22000000\n",
      "current_index: 23000000\n",
      "current_index: 24000000\n",
      "current_index: 25000000\n",
      "current_index: 26000000\n",
      "current_index: 27000000\n",
      "current_index: 28000000\n",
      "current_index: 29000000\n",
      "current_index: 30000000\n",
      "current_index: 31000000\n",
      "current_index: 32000000\n",
      "current_index: 33000000\n",
      "current_index: 34000000\n",
      "current_index: 35000000\n",
      "current_index: 36000000\n",
      "current_index: 37000000\n",
      "current_index: 38000000\n",
      "current_index: 39000000\n",
      "current_index: 40000000\n",
      "current_index: 41000000\n",
      "current_index: 42000000\n",
      "0\n"
     ]
    }
   ],
   "source": [
    "#-*- coding:utf-8 -*-\n",
    "f = open(\"/Users/maxiao/Documents/work@2018/重点项目/ESMM开源/sample_train/sample_skeleton_train.csv\",'r')   \n",
    "f_o = open(\"./ctr_cvr_data/BuyWeight_sample_skeleton_train_sample_2_percent.csv\",'w') \n",
    "train_md5_set = set()\n",
    "index = 0\n",
    "for line in f:\n",
    "    tokens = line.strip().split(\",\")\n",
    "    if tokens[2] == '1' or random.uniform(0, 1) < 0.025:\n",
    "        f_o.write(line)\n",
    "        train_md5_set.add(tokens[3])\n",
    "    index += 1\n",
    "    if index % 1000000 == 0:\n",
    "        print(\"current_index:\",index)\n",
    "f.close()\n",
    "f_o.close()\n",
    "pickle.dump(train_md5_set, open('./ctr_cvr_data/BuyWeight_train_md5_set.p', 'wb'))\n",
    "print(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "current_index: 1000000\n",
      "current_index: 2000000\n",
      "current_index: 3000000\n",
      "current_index: 4000000\n",
      "current_index: 5000000\n",
      "current_index: 6000000\n",
      "current_index: 7000000\n",
      "current_index: 8000000\n",
      "current_index: 9000000\n",
      "current_index: 10000000\n",
      "current_index: 11000000\n",
      "current_index: 12000000\n",
      "current_index: 13000000\n",
      "current_index: 14000000\n",
      "current_index: 15000000\n",
      "current_index: 16000000\n",
      "current_index: 17000000\n",
      "current_index: 18000000\n",
      "current_index: 19000000\n",
      "current_index: 20000000\n",
      "current_index: 21000000\n",
      "current_index: 22000000\n",
      "current_index: 23000000\n",
      "current_index: 24000000\n",
      "current_index: 25000000\n",
      "current_index: 26000000\n",
      "current_index: 27000000\n",
      "current_index: 28000000\n",
      "current_index: 29000000\n",
      "current_index: 30000000\n",
      "current_index: 31000000\n",
      "current_index: 32000000\n",
      "current_index: 33000000\n",
      "current_index: 34000000\n",
      "current_index: 35000000\n",
      "current_index: 36000000\n",
      "current_index: 37000000\n",
      "current_index: 38000000\n",
      "current_index: 39000000\n",
      "current_index: 40000000\n",
      "current_index: 41000000\n",
      "current_index: 42000000\n",
      "current_index: 43000000\n",
      "0\n"
     ]
    }
   ],
   "source": [
    "#-*- coding:utf-8 -*-\n",
    "f = open(\"/Users/maxiao/Documents/work@2018/重点项目/ESMM开源/sample_test/sample_skeleton_test.csv\",'r')   \n",
    "f_o = open(\"./ctr_cvr_data/BuyWeight_sample_skeleton_test_sample_2_percent.csv\",'w') \n",
    "test_md5_set = set()\n",
    "index = 0\n",
    "for line in f:\n",
    "    tokens = line.strip().split(\",\")\n",
    "    if tokens[2] == '1' or random.uniform(0, 1) < 0.025:\n",
    "        f_o.write(line)\n",
    "        test_md5_set.add(tokens[3])\n",
    "    index += 1\n",
    "    if index % 1000000 == 0:\n",
    "        print(\"current_index:\",index)\n",
    "f.close()\n",
    "f_o.close()\n",
    "pickle.dump(test_md5_set, open('./ctr_cvr_data/BuyWeight_test_md5_set.p', 'wb'))\n",
    "print(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "467621\n",
      "current_index: 10000\n",
      "current_index: 20000\n",
      "current_index: 30000\n",
      "current_index: 40000\n",
      "current_index: 50000\n",
      "current_index: 60000\n",
      "current_index: 70000\n",
      "current_index: 80000\n",
      "current_index: 90000\n",
      "current_index: 100000\n",
      "current_index: 110000\n",
      "current_index: 120000\n",
      "current_index: 130000\n",
      "current_index: 140000\n",
      "current_index: 150000\n",
      "current_index: 160000\n",
      "current_index: 170000\n",
      "current_index: 180000\n",
      "current_index: 190000\n",
      "current_index: 200000\n",
      "current_index: 210000\n",
      "current_index: 220000\n",
      "current_index: 230000\n",
      "current_index: 240000\n",
      "current_index: 250000\n",
      "current_index: 260000\n",
      "current_index: 270000\n",
      "current_index: 280000\n",
      "current_index: 290000\n",
      "current_index: 300000\n",
      "current_index: 310000\n",
      "current_index: 320000\n",
      "current_index: 330000\n",
      "current_index: 340000\n",
      "current_index: 350000\n",
      "current_index: 360000\n",
      "current_index: 370000\n",
      "current_index: 380000\n",
      "current_index: 390000\n",
      "current_index: 400000\n",
      "current_index: 410000\n",
      "current_index: 420000\n",
      "current_index: 430000\n",
      "current_index: 440000\n",
      "current_index: 450000\n",
      "current_index: 460000\n",
      "current_index: 470000\n",
      "current_index: 480000\n",
      "current_index: 490000\n",
      "current_index: 500000\n",
      "current_index: 510000\n",
      "current_index: 520000\n",
      "current_index: 530000\n",
      "current_index: 540000\n",
      "current_index: 550000\n",
      "current_index: 560000\n",
      "current_index: 570000\n",
      "current_index: 580000\n",
      "current_index: 590000\n",
      "current_index: 600000\n",
      "current_index: 610000\n",
      "current_index: 620000\n",
      "current_index: 630000\n",
      "current_index: 640000\n",
      "current_index: 650000\n",
      "current_index: 660000\n",
      "current_index: 670000\n",
      "current_index: 680000\n",
      "current_index: 690000\n",
      "current_index: 700000\n",
      "current_index: 710000\n",
      "current_index: 720000\n",
      "current_index: 730000\n",
      "0\n"
     ]
    }
   ],
   "source": [
    "train_md5_set = pickle.load(open('./ctr_cvr_data/BuyWeight_train_md5_set.p', mode='rb'))\n",
    "print(len(train_md5_set))\n",
    "#-*- coding:utf-8 -*-\n",
    "f = open(\"/Users/maxiao/Documents/work@2018/重点项目/ESMM开源/sample_train/common_features_train.csv\",'r')   \n",
    "f_o = open(\"./ctr_cvr_data/BuyWeight_common_features_skeleton_train_sample_2_percent.csv\",'w') \n",
    "\n",
    "index = 0\n",
    "for line in f:\n",
    "    tokens = line.strip().split(\",\")\n",
    "    value=0\n",
    "    md5 = tokens[0]\n",
    "    if md5 in train_md5_set:\n",
    "        f_o.write(line)\n",
    "    index += 1\n",
    "    if index % 10000 == 0:\n",
    "        print(\"current_index:\",index)\n",
    "f.close()\n",
    "f_o.close()\n",
    "\n",
    "print(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "541635\n",
      "current_index: 10000\n",
      "current_index: 20000\n",
      "current_index: 30000\n",
      "current_index: 40000\n",
      "current_index: 50000\n",
      "current_index: 60000\n",
      "current_index: 70000\n",
      "current_index: 80000\n",
      "current_index: 90000\n",
      "current_index: 100000\n",
      "current_index: 110000\n",
      "current_index: 120000\n",
      "current_index: 130000\n",
      "current_index: 140000\n",
      "current_index: 150000\n",
      "current_index: 160000\n",
      "current_index: 170000\n",
      "current_index: 180000\n",
      "current_index: 190000\n",
      "current_index: 200000\n",
      "current_index: 210000\n",
      "current_index: 220000\n",
      "current_index: 230000\n",
      "current_index: 240000\n",
      "current_index: 250000\n",
      "current_index: 260000\n",
      "current_index: 270000\n",
      "current_index: 280000\n",
      "current_index: 290000\n",
      "current_index: 300000\n",
      "current_index: 310000\n",
      "current_index: 320000\n",
      "current_index: 330000\n",
      "current_index: 340000\n",
      "current_index: 350000\n",
      "current_index: 360000\n",
      "current_index: 370000\n",
      "current_index: 380000\n",
      "current_index: 390000\n",
      "current_index: 400000\n",
      "current_index: 410000\n",
      "current_index: 420000\n",
      "current_index: 430000\n",
      "current_index: 440000\n",
      "current_index: 450000\n",
      "current_index: 460000\n",
      "current_index: 470000\n",
      "current_index: 480000\n",
      "current_index: 490000\n",
      "current_index: 500000\n",
      "current_index: 510000\n",
      "current_index: 520000\n",
      "current_index: 530000\n",
      "current_index: 540000\n",
      "current_index: 550000\n",
      "current_index: 560000\n",
      "current_index: 570000\n",
      "current_index: 580000\n",
      "current_index: 590000\n",
      "current_index: 600000\n",
      "current_index: 610000\n",
      "current_index: 620000\n",
      "current_index: 630000\n",
      "current_index: 640000\n",
      "current_index: 650000\n",
      "current_index: 660000\n",
      "current_index: 670000\n",
      "current_index: 680000\n",
      "current_index: 690000\n",
      "current_index: 700000\n",
      "current_index: 710000\n",
      "current_index: 720000\n",
      "current_index: 730000\n",
      "current_index: 740000\n",
      "current_index: 750000\n",
      "current_index: 760000\n",
      "current_index: 770000\n",
      "current_index: 780000\n",
      "current_index: 790000\n",
      "current_index: 800000\n",
      "current_index: 810000\n",
      "current_index: 820000\n",
      "current_index: 830000\n",
      "current_index: 840000\n",
      "current_index: 850000\n",
      "current_index: 860000\n",
      "current_index: 870000\n",
      "current_index: 880000\n",
      "0\n"
     ]
    }
   ],
   "source": [
    "test_md5_set = pickle.load(open('./ctr_cvr_data/BuyWeight_test_md5_set.p', mode='rb'))\n",
    "print(len(test_md5_set))\n",
    "\n",
    "#-*- coding:utf-8 -*-\n",
    "f = open(\"/Users/maxiao/Documents/work@2018/重点项目/ESMM开源/sample_test/common_features_test.csv\",'r')   \n",
    "f_o = open(\"./ctr_cvr_data/BuyWeight_common_features_skeleton_test_sample_2_percent.csv\",'w') \n",
    "\n",
    "index = 0\n",
    "for line in f:\n",
    "    tokens = line.strip().split(\",\")\n",
    "    value=0\n",
    "    md5 = tokens[0]\n",
    "    if md5 in test_md5_set:\n",
    "        f_o.write(line)\n",
    "    index += 1\n",
    "    if index % 10000 == 0:\n",
    "        print(\"current_index:\",index)\n",
    "f.close()\n",
    "f_o.close()\n",
    "\n",
    "print(0)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
